<!-- Web Scraping Ingest Tab -->
<div id="tabWebScrapingIngest" class="tab-content" role="tabpanel">
    <div class="endpoint-section" id="section_friendlyIngest">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/media/ingest-web-content</span>
        </h2>
        <p>Friendly form - no JSON required. Fields map to IngestWebContentRequest.</p>
        <div id="friendlyIngest_validation_summary" class="validation-summary" aria-live="polite"></div>

        <div class="columns">
            <div class="column">
                <h3>Sources</h3>
                <div class="form-group">
                    <label for="friendlyIngest_urls">URLs (one per line) <span class="required">*</span>:</label>
                    <textarea id="friendlyIngest_urls" rows="5" placeholder="https://example.com/article1&#10;https://example.com/article2"></textarea>
                    <small>Required. At least one URL.</small>
                    <small id="friendlyIngest_urls_hint" class="field-hint"></small>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_titles">Custom Titles (one per line, optional):</label>
                    <textarea id="friendlyIngest_titles" rows="3" placeholder="Custom Title 1&#10;Custom Title 2"></textarea>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_authors">Authors (one per line, optional):</label>
                    <textarea id="friendlyIngest_authors" rows="3" placeholder="Author 1&#10;Author 2"></textarea>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_keywords">Keywords (comma-separated):</label>
                    <input type="text" id="friendlyIngest_keywords" placeholder="web,scraping,example">
                </div>
            </div>
            <div class="column">
                <h3>Scraping Method</h3>
                <div class="form-group">
                    <label for="friendlyIngest_scrape_method">Method:
                        <span title="Methods:\n- individual: Scrapes each URL independently.\n- sitemap: Treats the first URL as a sitemap; uses Max Pages.\n- url_level: Scrapes pages under the base URL up to the given path level; requires URL Level.\n- recursive_scraping: Crawls links up to Max Depth and Max Pages.">
                            ⓘ
                        </span>
                    </label>
                    <select id="friendlyIngest_scrape_method">
                        <option value="individual" selected>individual (each URL)</option>
                        <option value="sitemap">sitemap (first URL is sitemap)</option>
                        <option value="url_level">url_level (collect by path depth)</option>
                        <option value="recursive_scraping">recursive_scraping (crawl links)</option>
                    </select>
                </div>
                <div class="form-group" id="group_friendlyIngest_url_level">
                    <label for="friendlyIngest_url_level">URL Level (for url_level):</label>
                    <input type="number" id="friendlyIngest_url_level" min="1" value="2">
                    <small id="friendlyIngest_url_level_hint" class="field-hint"></small>
                </div>
                <div class="form-group" id="group_friendlyIngest_max_pages">
                    <label for="friendlyIngest_max_pages">Max Pages (for recursive/sitemap):</label>
                    <input type="number" id="friendlyIngest_max_pages" min="1" value="10">
                    <small id="friendlyIngest_max_pages_hint" class="field-hint"></small>
                </div>
                <div class="form-group" id="group_friendlyIngest_max_depth">
                    <label for="friendlyIngest_max_depth">Max Depth (for recursive):</label>
                    <input type="number" id="friendlyIngest_max_depth" min="1" value="3">
                    <small id="friendlyIngest_max_depth_hint" class="field-hint"></small>
                </div>
                <!-- Crawl controls -->
                <div class="form-group" id="group_friendlyIngest_crawl_strategy">
                    <label for="friendlyIngest_crawl_strategy">Crawl Strategy:</label>
                    <select id="friendlyIngest_crawl_strategy">
                        <option value="best_first" selected>best_first (default)</option>
                        <option value="default">default (FIFO/BFS)</option>
                    </select>
                    <small>Applies to recursive/url_level when supported.</small>
                </div>
                <div class="form-group" id="group_friendlyIngest_include_external">
                    <label>
                        <input type="checkbox" id="friendlyIngest_include_external">
                        Include external domains
                    </label>
                </div>
                <div class="form-group" id="group_friendlyIngest_score_threshold">
                    <label for="friendlyIngest_score_threshold">Score Threshold:</label>
                    <input type="number" id="friendlyIngest_score_threshold" value="0" step="0.1" min="0">
                </div>
            </div>
            <div class="column">
                <h3>Analysis & Model</h3>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_perform_analysis" checked>
                        Perform Analysis
                    </label>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_custom_prompt">Custom Prompt:</label>
                    <textarea id="friendlyIngest_custom_prompt" rows="3" placeholder="Summarize with bullet points."></textarea>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_system_prompt">System Prompt:</label>
                    <textarea id="friendlyIngest_system_prompt" rows="3" placeholder="You are an accurate summarizer."></textarea>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_api_name">Model:</label>
                    <select id="friendlyIngest_api_name" class="llm-model-select">
                        <option value="">Use default</option>
                    </select>
                    <small>Populated from configured providers.</small>
                </div>
            </div>
        </div>

        <div class="columns">
            <div class="column">
                <h3>Translation</h3>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_perform_translation">
                        Perform Translation
                    </label>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_translation_language">Language:</label>
                    <input type="text" id="friendlyIngest_translation_language" value="en" placeholder="en">
                </div>
            </div>
            <div class="column">
                <h3>Chunking</h3>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_perform_chunking" checked>
                        Perform Chunking
                    </label>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_chunk_method">Method:</label>
                    <select id="friendlyIngest_chunk_method">
                        <option value="words" selected>words</option>
                        <option value="sentences">sentences</option>
                        <option value="paragraphs">paragraphs</option>
                        <option value="tokens">tokens</option>
                        <option value="semantic">semantic</option>
                        <option value="propositions">propositions</option>
                    </select>
                </div>
                <div class="form-group" id="friendlyIngest_prop_engine_group" style="display:none;">
                    <label for="friendlyIngest_prop_engine">Proposition Engine:</label>
                    <select id="friendlyIngest_prop_engine">
                        <option value="heuristic" selected>heuristic</option>
                        <option value="spacy">spacy</option>
                        <option value="llm">llm</option>
                        <option value="auto">auto</option>
                    </select>
                </div>
                <div class="form-group" id="friendlyIngest_prop_profile_group" style="display:none;">
                    <label for="friendlyIngest_prop_profile">Prompt Profile:</label>
                    <select id="friendlyIngest_prop_profile">
                        <option value="generic" selected>generic</option>
                        <option value="claimify">claimify</option>
                        <option value="gemma_aps">gemma_aps</option>
                    </select>
                </div>
                <div class="form-group" id="friendlyIngest_prop_aggr_group" style="display:none;">
                    <label for="friendlyIngest_prop_aggr">Aggressiveness (0-2):</label>
                    <input type="number" id="friendlyIngest_prop_aggr" value="1" min="0" max="2">
                </div>
                <div class="form-group" id="friendlyIngest_prop_minlen_group" style="display:none;">
                    <label for="friendlyIngest_prop_minlen">Min Proposition Length:</label>
                    <input type="number" id="friendlyIngest_prop_minlen" value="15" min="0">
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_chunk_size">Chunk Size:</label>
                    <input type="number" id="friendlyIngest_chunk_size" value="500" min="50">
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_chunk_overlap">Overlap:</label>
                    <input type="number" id="friendlyIngest_chunk_overlap" value="200" min="0">
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_use_adaptive_chunking">
                        Use Adaptive Chunking
                    </label>
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_use_multi_level_chunking">
                        Use Multi-Level Chunking
                    </label>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_chunk_language">Chunk Language:</label>
                    <input type="text" id="friendlyIngest_chunk_language" placeholder="en">
                </div>
            </div>
            <div class="column">
                <h3>Other Options</h3>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_use_cookies">
                        Use Cookies
                    </label>
                </div>
                <div class="form-group" id="group_friendlyIngest_cookies">
                    <label for="friendlyIngest_cookies">Cookies JSON (if using cookies):</label>
                    <textarea id="friendlyIngest_cookies" rows="4" placeholder='{"name":"mycookie","value":"abc","domain":".example.com"}'></textarea>
                    <small id="friendlyIngest_cookies_status" class="field-hint"></small>
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_timestamp_option" checked>
                        Add Timestamp
                    </label>
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_overwrite_existing">
                        Overwrite Existing
                    </label>
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_perform_rolling_summarization">
                        Rolling Summarization
                    </label>
                </div>
                <div class="form-group">
                    <label>
                        <input type="checkbox" id="friendlyIngest_perform_confabulation_check_of_analysis">
                        Confabulation Check
                    </label>
                </div>
                <div class="form-group">
                    <label for="friendlyIngest_custom_chapter_pattern">Custom Chapter Regex (optional):</label>
                    <input type="text" id="friendlyIngest_custom_chapter_pattern" placeholder="^(Chapter|Section) \\d+">
                </div>
            </div>
        </div>

        <div class="form-group">
            <button class="api-button" id="friendlyIngest_submit">Build & Submit</button>
            <button class="btn btn-secondary" id="friendlyIngest_show_curl">Show cURL</button>
        </div>
        <div id="friendlyIngest_validation_hint" aria-live="polite" style="color: var(--color-error, #c62828);"></div>
        <textarea id="friendlyIngest_payload" style="display:none;"></textarea>
        <pre id="friendlyIngest_curl" style="display:none;"></pre>
        <pre id="friendlyIngest_response">---</pre>
    </div>

    <div class="endpoint-section" id="webScrapingIngest">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/media/ingest-web-content</span>
        </h2>
        <p>Raw JSON builder (for reference). Identical to the form above.</p>

        <div class="form-group">
            <label for="webScrapingIngest_payload">Payload (IngestWebContentRequest JSON):</label>
            <textarea id="webScrapingIngest_payload" style="min-height: 360px;">{
  "urls": [
    "https://example.com/article1",
    "https://example.com/article2"
  ],
  "titles": [
    "Custom Title 1",
    "Custom Title 2"
  ],
  "authors": [
    "Author 1",
    "Author 2"
  ],
  "keywords": [
    "web",
    "scraping",
    "example"
  ],
  "scrape_method": "individual",
  "url_level": 2,
  "max_pages": 10,
  "max_depth": 3,

  "perform_analysis": true,
  "custom_prompt": "Summarize with bullet points.",
  "system_prompt": "You are an accurate summarizer.",
  "api_name": null,

  "perform_translation": false,
  "translation_language": "en",

  "perform_rolling_summarization": false,
  "perform_confabulation_check_of_analysis": false,

  "perform_chunking": true,
  "chunk_method": "words",
  "use_adaptive_chunking": false,
  "use_multi_level_chunking": false,
  "chunk_language": "en",
  "chunk_size": 500,
  "chunk_overlap": 200,

  "use_cookies": false,
  "cookies": null,

  "timestamp_option": true,
  "overwrite_existing": false,
  "custom_chapter_pattern": null
}</textarea>
            <small>
                Notes:
                - scrape_method: one of "individual", "sitemap", "url_level", "recursive_scraping".
                - When using "url_level" provide url_level; when "recursive_scraping" provide max_pages and max_depth.
                - API keys are managed server-side; do not include api_key here.
            </small>
        </div>

        <button class="api-button" id="btnWSIngestSubmit">
            Submit
        </button>
        <h3>cURL Command:</h3>
        <pre id="webScrapingIngest_curl">---</pre>
        <h3>Response:</h3>
        <pre id="webScrapingIngest_response">---</pre>
    </div>

    <div class="endpoint-section" id="webScrapingProcessLegacy">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/media/process-web-scraping</span>
        </h2>
        <p>Legacy-compatible web scraping endpoint that delegates to the enhanced service under the hood. Accepts multi-line URL input and supports persist/ephemeral modes.</p>

        <div class="form-group">
            <label for="webScrapingProcessLegacy_payload">Payload (WebScrapingRequest JSON):</label>
            <textarea id="webScrapingProcessLegacy_payload" style="min-height: 280px;">{
  "scrape_method": "Individual URLs",
  "url_input": "https://example.com/one\nhttps://example.com/two",
  "url_level": null,
  "max_pages": 10,
  "max_depth": 3,
  "summarize_checkbox": true,
  "custom_prompt": "Please summarize",
  "api_name": null,
  "keywords": "web,scraping,example",
  "custom_titles": "Title 1\nTitle 2",
  "system_prompt": null,
  "temperature": 0.7,
  "custom_cookies": [],
  "mode": "persist"
}</textarea>
            <small>
                scrape_method: one of "Individual URLs", "Sitemap", "URL Level", "Recursive Scraping". For "URL Level" provide url_level.
            </small>
        </div>

        <button class="api-button" id="btnWSLegacySubmit">
            Submit
        </button>
        <h3>cURL Command:</h3>
        <pre id="webScrapingProcessLegacy_curl">---</pre>
        <h3>Response:</h3>
        <pre id="webScrapingProcessLegacy_response">---</pre>
    </div>
</div>

<!-- Web Scraping Management Tab -->
<div id="tabWebScrapingStatus" class="tab-content" role="tabpanel">
    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method get">GET</span>
            <span class="endpoint-path">/api/v1/web-scraping/status</span>
        </h2>
        <p>Get web scraping service status and active jobs</p>

        <button class="api-button" id="btnWSStatus">
            Check Status
        </button>

        <pre id="webScrapingStatus_response"></pre>
    </div>
</div>

<div id="tabWebScrapingJobs" class="tab-content" role="tabpanel">
    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method get">GET</span>
            <span class="endpoint-path">/api/v1/web-scraping/job/{job_id}</span>
        </h2>
        <p>Get specific job details</p>

        <div class="form-group">
            <label for="webScrapingJobGet_job_id">Job ID <span class="required">*</span>:</label>
            <input type="text" id="webScrapingJobGet_job_id" placeholder="Enter job ID">
        </div>

        <button class="api-button" id="btnWSJobGet">Get Job Details</button>

        <pre id="webScrapingJobGet_response"></pre>
    </div>

    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method delete">DELETE</span>
            <span class="endpoint-path">/api/v1/web-scraping/job/{job_id}</span>
        </h2>
        <p>Cancel or delete a scraping job</p>

        <div class="form-group">
            <label for="webScrapingJobDelete_job_id">Job ID <span class="required">*</span>:</label>
            <input type="text" id="webScrapingJobDelete_job_id" placeholder="Enter job ID">
        </div>

        <button class="api-button btn-danger" id="btnWSJobDelete">Cancel Job</button>

        <pre id="webScrapingJobDelete_response"></pre>
    </div>

    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method get">GET</span>
            <span class="endpoint-path">/api/v1/web-scraping/progress/{task_id}</span>
        </h2>
        <p>Get task progress information</p>

        <div class="form-group">
            <label for="webScrapingProgress_task_id">Task ID <span class="required">*</span>:</label>
            <input type="text" id="webScrapingProgress_task_id" placeholder="Enter task ID">
        </div>

        <button class="api-button" id="btnWSProgress">Check Progress</button>

        <pre id="webScrapingProgress_response"></pre>
    </div>
</div>

<div id="tabWebScrapingService" class="tab-content" role="tabpanel">
    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/web-scraping/service/initialize</span>
        </h2>
        <p>Initialize the web scraping service.</p>
        <small>No request body required. Uses server configuration.</small>

        <button class="api-button" id="btnWSInit">Initialize Service</button>

        <pre id="webScrapingInit_response"></pre>
    </div>

    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/web-scraping/service/shutdown</span>
        </h2>
        <p>Shutdown the web scraping service.</p>
        <small>No request body required. Shuts down gracefully.</small>

        <button class="api-button btn-danger" id="btnWSShutdown">Shutdown Service</button>

        <pre id="webScrapingShutdown_response"></pre>
    </div>
</div>

<div id="tabWebScrapingCookies" class="tab-content" role="tabpanel">
    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method get">GET</span>
            <span class="endpoint-path">/api/v1/web-scraping/cookies/{domain}</span>
        </h2>
        <p>Get cookies for a specific domain</p>

        <div class="form-group">
            <label for="webScrapingCookiesGet_domain">Domain <span class="required">*</span>:</label>
            <input type="text" id="webScrapingCookiesGet_domain" placeholder="example.com">
        </div>

        <button class="api-button" id="btnWSCookiesGet">Get Cookies</button>

        <pre id="webScrapingCookiesGet_response"></pre>
    </div>

    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method post">POST</span>
            <span class="endpoint-path">/api/v1/web-scraping/cookies/{domain}</span>
        </h2>
        <p>Set cookies for a specific domain</p>

        <div class="form-group">
            <label for="webScrapingCookiesSet_domain">Domain <span class="required">*</span>:</label>
            <input type="text" id="webScrapingCookiesSet_domain" placeholder="example.com">
        </div>

        <div class="form-group">
            <label for="webScrapingCookiesSet_payload">Cookies (JSON):</label>
            <textarea id="webScrapingCookiesSet_payload" rows="8" placeholder='[{"name": "session", "value": "abc123", "domain": ".example.com"}]'>[
  {
    "name": "session_id",
    "value": "your_session_value",
    "domain": ".example.com",
    "path": "/",
    "secure": true,
    "httpOnly": true
  }
]</textarea>
        </div>

        <button class="api-button" id="btnWSCookiesSet">Set Cookies</button>

        <pre id="webScrapingCookiesSet_response"></pre>
    </div>
</div>

<div id="tabWebScrapingDuplicates" class="tab-content" role="tabpanel">
    <div class="endpoint-section">
        <h2>
            <span class="endpoint-method get">GET</span>
            <span class="endpoint-path">/api/v1/web-scraping/duplicates/check</span>
        </h2>
        <p>Check for duplicate URLs in the database</p>

        <div class="form-group">
            <label for="webScrapingDuplicates_url">URL to Check:</label>
            <input type="text" id="webScrapingDuplicates_url" placeholder="https://example.com/page" data-query="true">
        </div>

        <button class="api-button" id="btnWSDuplicates">Check Duplicates</button>

        <pre id="webScrapingDuplicates_response"></pre>
    </div>
</div>
